import pandas as pd
import pandas_profiling
import numpy as np
import seaborn as sns
import sklearn
import missingno as msno
import matplotlib.pyplot as plt
%matplotlib inline
import plotly.express as px
from mpl_toolkits.mplot3d import Axes3D
from sklearn import preprocessing
from sklearn import model_selection
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.preprocessing import StandardScaler
from sklearn.dummy import DummyClassifier
from sklearn import metrics
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from yellowbrick.classifier import ConfusionMatrix
from yellowbrick.classifier import ROCAUC
import sklearn.metrics
from sklearn.metrics import confusion_matrix, roc_auc_score
pd.set_option('display.max_columns', 500)
from imblearn.over_sampling import SMOTE
Cervical Cancer Risk Factors for Biopsy: This Dataset is Obtained from UCI Repository and kindly acknowledged!
This file contains a List of Risk Factors for Cervical Cancer leading to a Biopsy Examination!
About 11,000 new cases of invasive cervical cancer are diagnosed each year in the U.S. However, the number of new cervical cancer cases has been declining steadily over the past decades. Although it is the most preventable type of cancer, each year cervical cancer kills about 4,000 women in the U.S. and about 300,000 women worldwide. In the United States, cervical cancer mortality rates plunged by 74% from 1955 - 1992 thanks to increased screening and early detection with the Pap test. AGE Fifty percent of cervical cancer diagnoses occur in women ages 35 - 54, and about 20% occur in women over 65 years of age. The median age of diagnosis is 48 years. About 15% of women develop cervical cancer between the ages of 20 - 30. Cervical cancer is extremely rare in women younger than age 20. However, many young women become infected with multiple types of human papilloma virus, which then can increase their risk of getting cervical cancer in the future. Young women with early abnormal changes who do not have regular examinations are at high risk for localized cancer by the time they are age 40, and for invasive cancer by age 50. SOCIOECONOMIC AND ETHNIC FACTORS Although the rate of cervical cancer has declined among both Caucasian and African-American women over the past decades, it remains much more prevalent in African-Americans -- whose death rates are twice as high as Caucasian women. Hispanic American women have more than twice the risk of invasive cervical cancer as Caucasian women, also due to a lower rate of screening. These differences, however, are almost certainly due to social and economic differences. Numerous studies report that high poverty levels are linked with low screening rates. In addition, lack of health insurance, limited transportation, and language difficulties hinder a poor woman’s access to screening services. HIGH SEXUAL ACTIVITY Human papilloma virus (HPV) is the main risk factor for cervical cancer. In adults, the most important risk factor for HPV is sexual activity with an infected person. Women most at risk for cervical cancer are those with a history of multiple sexual partners, sexual intercourse at age 17 years or younger, or both. A woman who has never been sexually active has a very low risk for developing cervical cancer. Sexual activity with multiple partners increases the likelihood of many other sexually transmitted infections (chlamydia, gonorrhea, syphilis).Studies have found an association between chlamydia and cervical cancer risk, including the possibility that chlamydia may prolong HPV infection. FAMILY HISTORY Women have a higher risk of cervical cancer if they have a first-degree relative (mother, sister) who has had cervical cancer. USE OF ORAL CONTRACEPTIVES Studies have reported a strong association between cervical cancer and long-term use of oral contraception (OC). Women who take birth control pills for more than 5 - 10 years appear to have a much higher risk HPV infection (up to four times higher) than those who do not use OCs. (Women taking OCs for fewer than 5 years do not have a significantly higher risk.) The reasons for this risk from OC use are not entirely clear. Women who use OCs may be less likely to use a diaphragm, condoms, or other methods that offer some protection against sexual transmitted diseases, including HPV. Some research also suggests that the hormones in OCs might help the virus enter the genetic material of cervical cells. HAVING MANY CHILDREN Studies indicate that having many children increases the risk for developing cervical cancer, particularly in women infected with HPV. SMOKING Smoking is associated with a higher risk for precancerous changes (dysplasia) in the cervix and for progression to invasive cervical cancer, especially for women infected with HPV. IMMUNOSUPPRESSION Women with weak immune systems, (such as those with HIV / AIDS), are more susceptible to acquiring HPV. Immunocompromised patients are also at higher risk for having cervical precancer develop rapidly into invasive cancer. DIETHYLSTILBESTROL (DES) From 1938 - 1971, diethylstilbestrol (DES), an estrogen-related drug, was widely prescribed to pregnant women to help prevent miscarriages. The daughters of these women face a higher risk for cervical cancer. DES is no longer prsecribed.
Leitura dataset¶
df = pd.read_csv(r'C:\Users\gebra\Desktop\DS\Dataset\Cancer\kag_risk_factors_cervical_cancer.csv')
df.head()
Subistituir '?' por nan¶
df = df.replace('?', np.nan)
Visualizar valores duplicados¶
df.duplicated().value_counts()
df[df.duplicated()].head()
df[(df['Number of sexual partners'] == 3.0) & (df['Age'] == 34.0) ].head()
df.drop_duplicates(inplace = True)
Visualizar a quantidade de valores missing de cada coluna¶
msno.bar(df)
Converter o dtype para float¶
df = df.astype('float32')
df.info()
df.head()
Visualizar a coluna Biopsias¶
df['Biopsy'].value_counts()
UTI = sorted(df['Biopsy'].unique())
uti = []
for k in UTI:
soma = df[(df['Biopsy'] == k)]
uti.append(len(soma))
x = np.arange(len(uti))
labels = ['N-Biopsy', 'Biopsy']
plt.figure(figsize = (15,10))
plt.bar(x, uti, color=['g', 'r'])
ax = plt.gca()
plt.xticks(x, labels)
plt.legend()
plt.show()
fig = px.bar(x = x, y = uti)
fig.update_layout(
title="Numero de Biopsias",
xaxis_title="Resultado Biopsia",
yaxis_title="Numero total",
legend_title="Legend Title"
)
fig.show()
import plotly.graph_objects as go
colors = ['green', 'red']
fig = go.Figure([go.Bar(x=x,y=uti, marker_color=colors, width=[0.5, 0.5])])
fig.update_layout(
title="Numero de Biopsias",
xaxis_title="Resultado Biopsia",
yaxis_title="Numero total",
legend_title="Legend Title"
)
fig.show()
A coluna target está desbalanceada.¶
Tratando as duas colunas com a menor quantidade de dados¶
x = df["STDs: Time since last diagnosis"].notna() & df['Biopsy'] ==1
x.value_counts()
x = df["STDs: Time since first diagnosis"].notna() & df['Biopsy'] ==1
x.value_counts()
colunas = ['STDs: Time since first diagnosis', 'STDs: Time since last diagnosis']
df = df.drop(colunas, axis = 1)
Neste caso as duas colunas foram dropadas, pois possuem poucos valores preenchidos e não tem relação com a target¶
msno.bar(df)
x = df["STDs:HPV"].notna() & df['Biopsy'] ==1
x.value_counts()
Preenchendo valores missing com a mediana¶
mediana = df.median()
df = df.fillna(mediana)
msno.bar(df)
Dividir em X, y para poder divir em treino e teste¶
X = df.drop(columns="Biopsy")
y = df['Biopsy']
Tratamento de classe desbalanceada¶
sm = SMOTE(random_state=42)
X_res, y_res = sm.fit_resample(X, y)
Divisão em treino e teste¶
X_train, X_test, y_train, y_test = model_selection.train_test_split(X_res, y_res, test_size=0.3, random_state=42)
modelos = [DummyClassifier,
LogisticRegression,
DecisionTreeClassifier,
KNeighborsClassifier,
GaussianNB,
RandomForestClassifier,
SVC,
]
modelos2 = ["DummyClassifier",
"LogisticRegression",
"DecisionTreeClassifier",
"KNeighborsClassifier",
"GaussianNB",
"RandomForestClassifier",
"SVC"]
k = 0
for modelo in modelos:
clf = modelo()
clf.fit(X_train, y_train)
resultado = clf.score(X_test, y_test, sample_weight=None)
print (modelos2[k],":",resultado)
k += 1
k = 0
for modelo in modelos:
clf = modelo()
clf.fit(X_train, y_train)
prev = clf.predict(X_test)
print (modelos2[k], ":", "\n", confusion_matrix(y_test, prev), "\n")
k += 1